# This is a sample Python script.

# Press ⌃R to execute it or replace it with your code.
# Press Double ⇧ to search everywhere for classes, files, tool windows, actions, and settings.

import requests
import xlwt
import os
import random
import time


from requests.packages import urllib3
urllib3.disable_warnings()

os.environ['NO_PROXY'] = 'www.taptap.com'


# 1. crawler website
def get_id():
    print("getData...")
    head = {
        'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) "
                      "Chrome/92.0.4515.159 Safari/537.36 ",
        'cookie': "sajssdk_2015_cross_new_user=1; _ga=GA1.2.264205966.1629341421; _gid=GA1.2.1509666692.1629341421; "
                  "ACCOUNT_SID=hHbdtBIDVYLMuOlKDigYktCVdGGnROjZdwPVohHn; "
                  "sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2217b5c510d6e1647-08139a386b8f97-35667c03"
                  "-1296000-17b5c510d6f167d%22%2C%22%24device_id%22%3A%2217b5c510d6e1647-08139a386b8f97-35667c03"
                  "-1296000-17b5c510d6f167d%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4"
                  "%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22"
                  "%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6"
                  "%8E%A5%E6%89%93%E5%BC%80%22%7D%2C%22first_id%22%3A%22%22%7D; "
                  "acw_tc=2760827916293702222062879ec1c318e53726c4c0604c6fe614d1140bd7a2; "
                  "XSRF-TOKEN=eyJpdiI6ImJSWCthc1NFZXhcL0lzZ3BqZkY0ZHlRPT0iLCJ2YWx1ZSI6ImtXYlFNUDc0cWlUR2xLTUhiblp3UUpmMjh0ZkFlcVdnNGN0cHJBMkxMcDQ1bDBrOER1RUNUaHUya292SUlQTGtybStmT3FxMVhMNDBXZnh3bXlUUlFRPT0iLCJtYWMiOiI5MmU4ODkxZTU3Yjk4ODM3ZjY4NGVkZjEyZDE1M2I1NTFmZmMxNjAwYjc1YWFmYjVmNjg0N2IzOTA1ZmZiYzI4In0%3D; tap_sess=eyJpdiI6ImZtYUtyT3I3N1FyUHhjVnV2RDNPbUE9PSIsInZhbHVlIjoiOE81a3oyTVo4Q1NhYkp3V3NYdGQ3QVNESnF0amZwM21qMEFOTHd5U2h3ZnRyeDBlYWtXVEJPY0QzTUIrejh0VDNsT1wvS3hiYk45TWRwRmVWOXdRaGd3PT0iLCJtYWMiOiIyZTI5MTMyNWY2MDdiZTA1OWUwZWMxZDk2YWI4NWYzNjU1NDkwMjEwNmE2Njg4YjIzZmEzMWY1ODFjOGQ1NDMzIn0%3D "
    }
    baseurl = 'https://www.taptap.com/webapiv2/app-tag/v1/by-tag?_trackParams=%7B%22refererLogParams%22%3A%7B%7D%7D' \
              '&from='
    datalist = []
    burl = []
    id_list = []
    for i in range(0, 901):
        url = baseurl + str(i * 10) + '&limit=10&tag=%E7%AD%96%E7%95%A5&X-UA=V%3D1%26PN%3DWebApp%26LANG%3Dzh_CN' \
                                      '%26VN_CODE%3D39%26VN%3D0.1.0%26LOC%3DCN%26PLT%3DPC%26DS%3DAndroid%26UID' \
                                      '%3D078a3315-8d25-44be-b0f0-5eeba5d6f533%26DT%3DPC'
        print(url)
        burl.append(url)
    for url in burl:
        time.sleep(random.random() * 3)
        resp = requests.get(url, headers=head).json()
        # print(resp)
        info_list = resp.get('data').get('list')
        for info in info_list:
            g_id = info.get('id')
            print(g_id)
            id_list.append(g_id)
    print(id_list)
    return id_list


def get_info():
    id_list = get_id()
    head = {
        'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) "
                      "Chrome/92.0.4515.159 Safari/537.36",
        'cookie': "sajssdk_2015_cross_new_user=1; _ga=GA1.2.264205966.1629341421; _gid=GA1.2.1509666692.1629341421; "
                  "ACCOUNT_SID=hHbdtBIDVYLMuOlKDigYktCVdGGnROjZdwPVohHn; "
                  "sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2217b5c510d6e1647-08139a386b8f97-35667c03"
                  "-1296000-17b5c510d6f167d%22%2C%22%24device_id%22%3A%2217b5c510d6e1647-08139a386b8f97-35667c03"
                  "-1296000-17b5c510d6f167d%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4"
                  "%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22"
                  "%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6"
                  "%8E%A5%E6%89%93%E5%BC%80%22%7D%2C%22first_id%22%3A%22%22%7D; "
                  "acw_tc=2760828416293500332138795e41a8003cfa1cee0c07861d624552465e20d4; "
                  "XSRF-TOKEN=eyJpdiI6ImpFNDZNOVZrTFBtS3prK2piV01Gb2c9PSIsInZhbHVlIjoiZE4xRzJjS0tXUUE2SEtvZ1pVakQzZ3ZFb2xNZVwvMVBcL2c4UG9xY0xEa0F1N29qV2tNWW1XbXFFV0l0aFl5ZEd4S3k1Z3c5REFkNVlGdlo3WEhmVGxwQT09IiwibWFjIjoiNGIwOTBjOGYyYWM1Y2ZmMjcxYmYzOGQ0ZTNlYjA0NjMxM2E3NmNhMmEwNDgzMTdiNzJjN2UxNzU1YjFlYzQxYSJ9; tap_sess=eyJpdiI6Ijg2emMrTUZ3Z2JFOHFoMWI1dGlvWXc9PSIsInZhbHVlIjoiQUZnSW9hZmMxakRoeFY5S0Yzem9yUExOT3JxNWxZdHU1WTF4RU9qckJ6Y1creVJVQ0pOU1hxSVpxMnhwRk9VSk5GNFlJcUpnVmdcL1wvZkZuZlwvMHFEQ2c9PSIsIm1hYyI6IjlkMmVlZDhmNTRkZDU5ZGUxZTgwYmQyZGZiYmMzYTI3ZjRmYzM2MzVkMjQ5YWM5ZWEzMWRkMjEzMTE5NzE3MWEifQ%3D%3D; _gat=1 "
    }
    baseurl = 'https://www.taptap.com/webapiv2/app/v2/detail-by-id/'
    info_url = []
    datalist = []
    for g_id in id_list:
        suburl = '?X-UA=V%3D1%26PN%3DWebApp%26LANG%3Dzh_CN%26VN_CODE%3D39%26VN%3D0.1.0%26LOC%3DCN%26PLT%3DPC%26DS' \
                 '%3DAndroid%26UID%3D078a3315-8d25-44be-b0f0-5eeba5d6f533%26DT%3DPC'
        i_url = baseurl + str(g_id) + suburl
        print(i_url)
        info_url.append(i_url)
    for url in info_url:
        data_all = []
        time.sleep(random.random() * 3)
        resp = requests.get(url, headers=head).json()
        # print(resp)
        data = resp.get('data')

        # title
        title = data.get('title')
        data_all.append(title)
        # print(data_all)

        # developers label
        developers = data.get('developers')
        if developers is not None:
            for dev in developers:
                label = dev.get('label')
                name = dev.get('name')
                developer = str(label + ": " + name)
                data_all.append(developer)
        # print(data_all)

        # print(len(data_all))
        if len(data_all) < 3:
            data_all.append(' ')

        # developer_website
        developer_website = data.get('developer_website')
        if developer_website is not None:
            data_all.append(developer_website)
        else:
            data_all.append(' ')
        # print(data_all)

        # stat -> fans_count (following)
        fans_count = data.get('stat').get('fans_count')
        if fans_count is not None:
            data_all.append(fans_count)
        else:
            data_all.append(' ')
        # print(data_all)

        # stat -> hits_total (download)
        hits_total = data.get('stat').get('hits_total')
        if hits_total is not None:
            data_all.append(hits_total)
        else:
            data_all.append(' ')
        # print(data_all)

        # stat -> rating -> score
        score = data.get('stat').get('rating').get('score')
        data_all.append(score)

        # stat -> review_count
        review_count = data.get('stat').get('review_count')
        if review_count is not None:
            data_all.append(review_count)
        else:
            data_all.append(' ')

        # developer_note
        if data.get('developer_note') is not None:
            developer_note = data.get('developer_note').get('text')
            developer_note = str(developer_note).replace('<br/>', '')
            developer_note = str(developer_note).replace('<br class="bbcode-paragraph-br"/>', '')
            data_all.append(developer_note)
        else:
            data_all.append(' ')

        # description
        if data.get('description') is not None:
            description = data.get('description').get('text')
            description = str(description).replace('<br/>', '')
            description = str(description).replace('<br class="bbcode-paragraph-br"/>', '')
            data_all.append(description)
        else:
            data_all.append(' ')

        # tags
        tags = data.get('tags')
        tags_list = []
        for tag in tags:
            value = tag.get('value')
            tags_list.append(value)
        data_all.append(str(tags_list))
        print(data_all)

        datalist.append(data_all)
    print(datalist)
    return datalist


def savedata(savepath):
    print("save data...")
    datalist = get_info()

    book = xlwt.Workbook(encoding='utf-8')

    sheet = book.add_sheet('Strategy_Taptap', cell_overwrite_ok=True)

    col = ("title", "developers", "developers", "developer_website", "following", "download", "score", "comment", "developer_note", "description", "tags")

    for i in range(0, 11):
        sheet.write(0, i, col[i])
    for i in range(0, len(datalist)):
        print("The %d item.." % (i + 1))
        data = datalist[i]
        # print(len(data))
        for j in range(0, len(data)):
            sheet.write(i+1, j, data[j])

    book.save(savepath)


def main():
    # Use a breakpoint in the code line below to debug your script.
    savepath = 'result/result_All.xls'
    savedata(savepath)


# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    main()

# See PyCharm help at https://www.jetbrains.com/help/pycharm/
